# -*- coding: utf-8 -*-

from PyQt4 import QtGui
from PyQt4.QtCore import QCoreApplication, QEventLoop
import sys, os, shutil, time, getopt, urllib2, time, re
import sqlite3 as sqlite
from editBugKeys import editBugKeys
from datetime import datetime, date, timedelta
from subprocess import *

projectName = ""
projectVersion = ""

projectStartDate = ""
projectEndDate = ""

trainRevisionInfo = ""

trainStartDate = ""
trainEndDate = ""

defaultDirectory = ""

connection = sqlite.connect('commits.db')
cursor = connection.cursor()

def listProjects1(browser):
    browser.clear()

    cursor.execute('CREATE Table if not exists projectTable (ID INTEGER PRIMARY KEY, projectName VARCHAR(50), projectVersion DOUBLE(5), projectStartDate DATETIME(20), projectEndDate DATETIME(20), projectNumberOfFiles INT(5), projectTotalLoc BIGINT(10))')
    cursor.execute('CREATE Table if not exists authorTable (ID INTEGER PRIMARY KEY, projectName VARCHAR(50), projectVersion DOUBLE(5), authorName VARCHAR(50), authorLinesOfCode INT(5))')
    cursor.execute('CREATE Table if not exists commitTable (ID INTEGER PRIMARY KEY, projectName VARCHAR(50), projectVersion DOUBLE(5), revisionNo BIGINT(10), authorName VARCHAR(20), commitDate DATETIME(20), commitMessage TEXT(1000))')
    cursor.execute('CREATE Table if not exists fileTable (ID INTEGER PRIMARY KEY, projectName VARCHAR(50), projectVersion DOUBLE(5), revisionNo BIGINT(10), commitDate DATETIME(20), authorName VARCHAR(50), filePath VARCHAR(200), fileLocAdd INT(5), fileLocRem INT(5))')
    
    cursor.execute('SELECT projectName, projectVersion FROM projectTable')
    projCount = cursor.fetchall()

    if len(projCount) > 0:
        browser.append("--------------------     PROJECTS IN THE DATABASE     --------------------\n")
        QCoreApplication.processEvents(QEventLoop.ExcludeUserInputEvents)
        i = 1
        for row in projCount:
            browser.append(str(i) + ") Project Name: " + row[0] + "\tVersion Name: " + str(row[1]))
            QCoreApplication.processEvents(QEventLoop.ExcludeUserInputEvents)
            i += 1
            
    elif len(projCount) == 0:
        browser.append("--------------------     THERE IS NO PROJECT IN THE DATABASE     --------------------\n")
        QCoreApplication.processEvents(QEventLoop.ExcludeUserInputEvents)

    browser.append("\nEnter a projectName and a projectVersion to store log info...")
    QCoreApplication.processEvents(QEventLoop.ExcludeUserInputEvents)

def listProjects2(browser):
    browser.clear()
    
    browser.append("--------------------     PROJECTS IN THE DATABASE     --------------------\n")
    QCoreApplication.processEvents(QEventLoop.ExcludeUserInputEvents)
    
    cursor.execute('SELECT projectName, projectVersion FROM projectTable')
    i = 1
    for row in cursor:
        browser.append(str(i) + ") Project Name: " + row[0] + "\tVersion Name: " + str(row[1]))
        QCoreApplication.processEvents(QEventLoop.ExcludeUserInputEvents)
        i += 1

    browser.append("\n\nEnter a projectName and a projectVersion to create Train & Test Csv Files...")
    QCoreApplication.processEvents(QEventLoop.ExcludeUserInputEvents)
    
def createProject(projName, projVersion, browser):
    browser.clear()
    
    projectName = str(projName)
    projectVersion = str(projVersion)

    sqliteSTR = "SELECT * FROM projectTable where projectName='" + projectName.strip() + "' and projectVersion = '" + projectVersion + "'"
    cursor.execute(sqliteSTR)
    projectExists = cursor.fetchall()

    if len(projectExists) == 1:
        print "Check the Project Name and the Project Version...\nYou entered an existing Project Name and a Project Version...\n"
        QCoreApplication.processEvents(QEventLoop.ExcludeUserInputEvents)
        sys.exit(1)

    return 5555

def pickProject(projName, projVersion, browser):
    browser.clear()
    
    global projectName
    global projectVersion

    projectName = str(projName)
    projectVersion = str(projVersion)
    
    sqliteSTR = "SELECT * FROM projectTable where projectName='" + projectName.strip() + "' and projectVersion = '" + projectVersion + "'"
    cursor.execute(sqliteSTR)
    projectExists = cursor.fetchall()

    if not len(projectExists) == 1:
        print "Check the Project Name or the Project Version...\nSelect a correct Project Name and a Project Version...\n" 
        QCoreApplication.processEvents(QEventLoop.ExcludeUserInputEvents)
        sys.exit(1)

    sqliteSTR = "SELECT * FROM projectTable where projectName='" + projectName.strip() + "' and projectVersion = '" + projectVersion + "'"
    cursor.execute(sqliteSTR)

    for row in cursor:
        projectStartDate = row[3]
        projectEndDate = row[4]

    browser.append("Details for the PROJECT: " + projectName + " " + str(projectVersion))
    print "Details for the PROJECT: " + projectName + " " + str(projectVersion)
    QCoreApplication.processEvents(QEventLoop.ExcludeUserInputEvents)
    browser.append("\nCOMMITS STARTED ON: " + projectStartDate)
    print "\nCOMMITS STARTED ON: " + projectStartDate
    QCoreApplication.processEvents(QEventLoop.ExcludeUserInputEvents)
    browser.append("COMMITS ENDED ON: " + projectEndDate)
    print "COMMITS ENDED ON: " + projectEndDate
    QCoreApplication.processEvents(QEventLoop.ExcludeUserInputEvents)

    temp1  = projectStartDate.split(" ")
    pStartDate = temp1[0]
    temp2  = projectEndDate.split(" ")
    pEndDate = temp2[0]

    projectStartDate = datetime.strptime(pStartDate, '%Y-%m-%d')
    projectEndDate = datetime.strptime(pEndDate, '%Y-%m-%d')

    return 5555

def createTrain(browser, start, end, last):
    global defaultDirectory
    defaultDirectory = os.getcwd()
    
    rootDir = "projectRoot"
    os.chdir("./" + rootDir + "/")
    
    sqliteSTR = "SELECT * FROM projectTable where projectName='" + projectName.strip() + "' and projectVersion = '" + projectVersion + "'"
    cursor.execute(sqliteSTR)

    for row in cursor:
        projectStartDate = row[3]
        projectEndDate = row[4]

    temp1  = projectStartDate.split(" ")
    pStartDate = temp1[0]
    temp2  = projectEndDate.split(" ")
    pEndDate = temp2[0]
    
    projectStartDate = datetime.strptime(pStartDate, '%Y-%m-%d')
    projectEndDate = datetime.strptime(pEndDate, '%Y-%m-%d')
    
    #create a project folder

    dirName = projectName
    if not os.path.isdir("./" + dirName + "/"):
        os.mkdir("./" + dirName + "/")
    os.chdir("./" + dirName + "/")

    versionName = projectVersion
    if os.path.isdir("./" + versionName + "/"):
        shutil.rmtree("./" + versionName + "/")
    if not os.path.isdir("./" + versionName + "/"):
        os.mkdir("./" + versionName + "/")
    os.chdir("./" + versionName + "/")


    startDate = datetime.strptime(str(start), '%Y-%m-%d')
    endDate = datetime.strptime(str(end), '%Y-%m-%d')
    lastReleaseDate = datetime.strptime(str(last), '%Y-%m-%d')

    global trainStartDate
    global trainEndDate

    trainStartDate = str(startDate)
    trainEndDate = str(endDate)

    print "\n<< Train's EndDate: " + str(trainEndDate) + " >>"
    
    compareDates = endDate - startDate
    if compareDates.days < 0:
        print "Check the DATES you entered...\nEND DATE must be later than START DATE"
        QCoreApplication.processEvents(QEventLoop.ExcludeUserInputEvents)
        sys.exit(1)    

    compareDates = lastReleaseDate - startDate
    if compareDates.days < 0:
        print "Check the DATES you entered...\nLAST RELEASE DATE must be later than START DATE"
        QCoreApplication.processEvents(QEventLoop.ExcludeUserInputEvents)
        sys.exit(1)

    compareDates = endDate - lastReleaseDate
    if compareDates.days < 0:
        print "Check the DATES you entered...\nEND DATE must be later than LAST RELEASE DATE"
        QCoreApplication.processEvents(QEventLoop.ExcludeUserInputEvents)
        sys.exit(1)    

    if projectEndDate < startDate:
        print "Check the DATES you entered..."
        QCoreApplication.processEvents(QEventLoop.ExcludeUserInputEvents)
        sys.exit(1)
        
    elif projectEndDate < startDate:
        if startDate < projectStartDate:
            startDate = projectStartDate

    if projectEndDate < endDate:
        endDate = projectEndDate

    global trainRevisionInfo    

    tempDate = endDate
    tempDayStart = tempDate
    tempDayStart = tempDayStart - timedelta(minutes = 1)
    
    tempDayEnd = tempDate    
    tempDayEnd = tempDayEnd + timedelta(hours = 23 )
    tempDayEnd = tempDayEnd + timedelta(minutes = 59 )
    tempDayEnd = tempDayEnd + timedelta(seconds = 59 )    

    sqliteSTR = "SELECT revisionNo from commitTable where projectName='" + projectName + "' and projectVersion='" + projectVersion + "' and commitDate between '" + str(tempDayStart) + "' and '" + str(tempDayEnd) + "' order by commitDate"
    cursor.execute(sqliteSTR)
    hasRevision = 0
    for row in cursor:
        hasRevision = hasRevision + 1
        trainRevisionInfo = row[0]

    while(hasRevision == 0):
        tempDayStart = tempDayStart - timedelta(days = 1)
        tempDayEnd = tempDayEnd - timedelta(days = 1)
        sqliteSTR = "SELECT revisionNo from commitTable where projectName='" + projectName + "' and projectVersion='" + projectVersion + "' and commitDate between '" + str(tempDayStart) + "' and '" + str(tempDayEnd) + "' order by commitDate"
        cursor.execute(sqliteSTR)

        for row in cursor:
            trainRevisionInfo = row[0]
            hasRevision = hasRevision + 1
    
    #create a train folder here and do file based operations

    dirName = "train"
    if not os.path.isdir("./" + dirName + "/"):
        os.mkdir("./" + dirName + "/")
    os.chdir("./" + dirName + "/")


    #create train files here

    trainDict = {}

    filename = "train.csv"    
    dataToWrite = "File Name,Number Of Commits,Number Of Committers,Lines Added,Lines Removed,Number Of Commits(LR),Number Of Committers(LR),Lines Added(LR),Lines Removed(LR),Gini Inequality,Top Committer Percentage\n"

    sqliteSTR = "SELECT filePath, fileLocAdd, fileLocRem from fileTable where projectName='" + projectName + "' and projectVersion='" + projectVersion + "' and commitDate between '" + str(startDate) + "' and '" + str(endDate) + "'"
    cursor.execute(sqliteSTR)

    for row in cursor:
        if row[0] in trainDict:        
            trainDict[row[0]]["added"] += row[1] # if key already exists, sum
            trainDict[row[0]]["removed"] += row[2] # if key already exists, sum
        else:
            trainDict[row[0]] = {}
            trainDict[row[0]]["added"] = row[1] # else create a new key
            trainDict[row[0]]["removed"] = row[2] # else create a new key

    browser.append("\n")
    browser.append("<< train.csv file creation starts... >>\n")
    for key in sorted(trainDict.keys()):
        dataToWrite += key #FILENAME
        browser.append("PROCESSING:\t" + key + "...")

        QCoreApplication.processEvents(QEventLoop.ExcludeUserInputEvents)
        sqliteSTR = "SELECT count(distinct(revisionNo)) from fileTable where projectName='" + projectName + "' and projectVersion='" + projectVersion + "' and filePath = '" + key + "' and commitDate between '" + str(startDate) + "' and '" + str(endDate) + "'"
        cursor.execute(sqliteSTR)
        for row in cursor:
            numberOfCommits = row[0]
            dataToWrite += "," + str(numberOfCommits) #1. Commits(Com) ! Number of commits on file cummulatively

        sqliteSTR = "SELECT count(distinct(authorName)) from fileTable where projectName='" + projectName + "' and projectVersion='" + projectVersion + "' and filePath = '" + key + "' and commitDate between '" + str(startDate) + "' and '" + str(endDate) + "'"
        cursor.execute(sqliteSTR)    
        for row in cursor:
            numberOfCommitters = row[0]
            dataToWrite += "," + str(numberOfCommitters) #2. Committers (Ctr) ! Number of committers on file cumulatively

        dataToWrite += "," + str(trainDict[key]["added"]) #3. Lines Added (Lad)!Lines added cummulatively
        dataToWrite += "," + str(trainDict[key]["removed"]) #4. Lines Removed (Lrm)!Lines removed cummulatively

        sqliteSTR = "SELECT count(distinct(revisionNo)) from fileTable where projectName='" + projectName + "' and projectVersion='" + projectVersion + "' and filePath = '" + key + "' and commitDate between '" + str(lastReleaseDate) + "' and '" + str(endDate) + "'"
        cursor.execute(sqliteSTR)
        for row in cursor:
            numberOfCommitsLR = row[0]
            dataToWrite += "," + str(numberOfCommitsLR) #5. Commits Last Release (ComLR)!Number of commits for last release

        sqliteSTR = "SELECT count(distinct(authorName)) from fileTable where projectName='" + projectName + "' and projectVersion='" + projectVersion + "' and filePath = '" + key + "' and commitDate between '" + str(lastReleaseDate) + "' and '" + str(endDate) + "'"
        cursor.execute(sqliteSTR)    
        for row in cursor:
            numberOfCommittersLR = row[0]
            dataToWrite += "," + str(numberOfCommittersLR) #6. Committers Last Release (CtrLR) ! Number of committers on files for last release

        sqliteSTR = "SELECT filePath, fileLocAdd, fileLocRem from fileTable where projectName='" + projectName + "' and projectVersion='" + projectVersion + "' and commitDate between '" + str(lastReleaseDate) + "' and '" + str(endDate) + "'"
        cursor.execute(sqliteSTR)

        trainDictLR = {}
        for row in cursor:
            if row[0] in trainDictLR:
                trainDictLR[row[0]]["added"] += row[1] # if key already exists, sum
                trainDictLR[row[0]]["removed"] += row[2] # if key already exists, sum
            else:
                trainDictLR[row[0]] = {}
                trainDictLR[row[0]]["added"] = row[1] # else create a new key
                trainDictLR[row[0]]["removed"] = row[2] # else create a new key

        if key in trainDictLR:
            dataToWrite += "," + str(trainDictLR[key]["added"]) #7. Lines Added Last Release (LadLR) ! Lines added on files for last release
        if key not in trainDictLR:
            dataToWrite += "," + str(0)
        
        if key in trainDictLR:
            dataToWrite += "," + str(trainDictLR[key]["removed"]) #8. Lines Removed Last Release (LrmLR) ! Lines added on files for last release
        if key not in trainDictLR:
            dataToWrite += "," + str(0)
        
        events = []
        sqliteSTR = "SELECT distinct(authorName) from fileTable where projectName='" + projectName + "' and projectVersion='" + projectVersion + "' and filePath = '" + key + "' and commitDate between '" + str(startDate) + "' and '" + str(endDate) + "'"
        cursor.execute(sqliteSTR)
        for row in cursor:
            events.append(row[0])

        occurrences = []
        sqliteSTR = "SELECT authorName from fileTable where projectName='" + projectName + "' and projectVersion='" + projectVersion + "' and filePath = '" + key + "' and commitDate between '" + str(startDate) + "' and '" + str(endDate) + "'"
        cursor.execute(sqliteSTR)    
        for row in cursor:
            occurrences.append(row[0])

        gini = 0    
        count = []    

        for i in range(len(events)):
            count.append(occurrences.count(events[i]))
    
        mean = len(occurrences) / float(len(events)) 
    
        for i in range(len(events)):
            for j in range(len(events)):
                gini += ( 1 / float(len(events) * len(events)))*abs( count[i] - count[j])
        giniIneq = gini/float(mean)

        dataToWrite += ",%.3f" % giniIneq #9. Gini Inequality Coefficient (GINI)!Commit inequality coefficient of committers on files

        if numberOfCommitters == 1:              #10. Top Committer Percentage (TCPer) ! Percentage of top committers on files
            dataToWrite += "," + str(100)
        elif not numberOfCommitters == 1:
            topCommiterPercentage = {}
            sqliteSTR = "SELECT authorName from fileTable where projectName='" + projectName + "' and projectVersion='" + projectVersion + "' and filePath = '" + key + "' and commitDate between '" + str(startDate) + "' and '" + str(endDate) + "'"
            cursor.execute(sqliteSTR)

            for row in cursor:
                if row[0] in topCommiterPercentage:
                    topCommiterPercentage[row[0]] += 1 # if key already exists, sum
                else:
                    topCommiterPercentage[row[0]] = 1 # else create a new key

            topCommiterCommit = 0

            for key in topCommiterPercentage:
                if topCommiterPercentage[key] > topCommiterCommit:
                    topCommiterCommit = topCommiterPercentage[key]
    
            dataToWrite += ",%.3f" % ((float(topCommiterCommit)/float(numberOfCommits))*100)            

        dataToWrite += "\n"

    FILE = open(filename,"w")
    FILE.writelines(dataToWrite)
    FILE.close()

    browser.append("\nThe train.csv File is CREATED...")
    QCoreApplication.processEvents(QEventLoop.ExcludeUserInputEvents)

    browser.append("\n\nTrain's EndDate Revision: " + str(trainRevisionInfo))
    QCoreApplication.processEvents(QEventLoop.ExcludeUserInputEvents)
    
    os.chdir("../")

    return trainRevisionInfo
    
def createTest(browser, start, end, last):
    global defaultDirectory
    defaultDirectory = os.getcwd()
    
    sqliteSTR = "SELECT * FROM projectTable where projectName='" + projectName.strip() + "' and projectVersion = '" + projectVersion + "'"
    cursor.execute(sqliteSTR)

    for row in cursor:
        projectStartDate = row[3]
        projectEndDate = row[4]

    temp1  = projectStartDate.split(" ")
    pStartDate = temp1[0]
    temp2  = projectEndDate.split(" ")
    pEndDate = temp2[0]
    
    projectStartDate = datetime.strptime(pStartDate, '%Y-%m-%d')
    projectEndDate = datetime.strptime(pEndDate, '%Y-%m-%d')
    
    startDate = datetime.strptime(str(start), '%Y-%m-%d')
    endDate = datetime.strptime(str(end), '%Y-%m-%d')
    lastReleaseDate = datetime.strptime(str(last), '%Y-%m-%d')

    compareDates = endDate - startDate
    if compareDates.days < 0:
        print "Check the DATES you entered...\nEND DATE must be later than START DATE"
        QCoreApplication.processEvents(QEventLoop.ExcludeUserInputEvents)
        sys.exit(1)    

    compareDates = lastReleaseDate - startDate
    if compareDates.days < 0:
        print "Check the DATES you entered...\nLAST RELEASE DATE must be later than START DATE"
        QCoreApplication.processEvents(QEventLoop.ExcludeUserInputEvents)
        sys.exit(1)

    compareDates = endDate - lastReleaseDate
    if compareDates.days < 0:
        print "Check the DATES you entered...\nEND DATE must be later than LAST RELEASE DATE"
        QCoreApplication.processEvents(QEventLoop.ExcludeUserInputEvents)
        sys.exit(1)    

    if projectEndDate < startDate:
        print "Check the DATES you entered..."
        QCoreApplication.processEvents(QEventLoop.ExcludeUserInputEvents)
        sys.exit(1)
        
    elif projectEndDate < startDate:
        if startDate < projectStartDate:
            startDate = projectStartDate

    if projectEndDate < endDate:
        endDate = projectEndDate


    #create a test folder here and do file based operations

    dirName = "test"
    if not os.path.isdir("./" + dirName + "/"):
        os.mkdir("./" + dirName + "/")
    os.chdir("./" + dirName + "/")


    #create test files here

    testDict = {}

    filename = "test.csv"    
    dataToWrite = "File Name,Number Of Commits,Number Of Committers,Lines Added,Lines Removed,Number Of Commits(LR),Number Of Committers(LR),Lines Added(LR),Lines Removed(LR),Gini Inequality,Top Committer Percentage\n"

    sqliteSTR = "SELECT filePath, fileLocAdd, fileLocRem from fileTable where projectName='" + projectName + "' and projectVersion='" + projectVersion + "' and commitDate between '" + str(startDate) + "' and '" + str(endDate) + "'"
    cursor.execute(sqliteSTR)

    for row in cursor:
        if row[0] in testDict:        
            testDict[row[0]]["added"] += row[1] # if key already exists, sum
            testDict[row[0]]["removed"] += row[2] # if key already exists, sum
        else:
            testDict[row[0]] = {}
            testDict[row[0]]["added"] = row[1] # else create a new key
            testDict[row[0]]["removed"] = row[2] # else create a new key

    browser.append("\n")
    browser.append("<< test.csv file creation starts... >>\n")
    for key in sorted(testDict.keys()):
        dataToWrite += key #FILENAME
        browser.append("PROCESSING:\t" + key + "...")

        QCoreApplication.processEvents(QEventLoop.ExcludeUserInputEvents)
        sqliteSTR = "SELECT count(distinct(revisionNo)) from fileTable where projectName='" + projectName + "' and projectVersion='" + projectVersion + "' and filePath = '" + key + "' and commitDate between '" + str(startDate) + "' and '" + str(endDate) + "'"
        cursor.execute(sqliteSTR)
        for row in cursor:
            numberOfCommits = row[0]
            dataToWrite += "," + str(numberOfCommits) #1. Commits(Com) ! Number of commits on file cummulatively

        sqliteSTR = "SELECT count(distinct(authorName)) from fileTable where projectName='" + projectName + "' and projectVersion='" + projectVersion + "' and filePath = '" + key + "' and commitDate between '" + str(startDate) + "' and '" + str(endDate) + "'"
        cursor.execute(sqliteSTR)    
        for row in cursor:
            numberOfCommitters = row[0]
            dataToWrite += "," + str(numberOfCommitters) #2. Committers (Ctr) ! Number of committers on file cumulatively

        dataToWrite += "," + str(testDict[key]["added"]) #3. Lines Added (Lad)!Lines added cummulatively
        dataToWrite += "," + str(testDict[key]["removed"]) #4. Lines Removed (Lrm)!Lines removed cummulatively

        sqliteSTR = "SELECT count(distinct(revisionNo)) from fileTable where projectName='" + projectName + "' and projectVersion='" + projectVersion + "' and filePath = '" + key + "' and commitDate between '" + str(lastReleaseDate) + "' and '" + str(endDate) + "'"
        cursor.execute(sqliteSTR)
        for row in cursor:
            numberOfCommitsLR = row[0]
            dataToWrite += "," + str(numberOfCommitsLR) #5. Commits Last Release (ComLR)!Number of commits for last release

        sqliteSTR = "SELECT count(distinct(authorName)) from fileTable where projectName='" + projectName + "' and projectVersion='" + projectVersion + "' and filePath = '" + key + "' and commitDate between '" + str(lastReleaseDate) + "' and '" + str(endDate) + "'"
        cursor.execute(sqliteSTR)    
        for row in cursor:
            numberOfCommittersLR = row[0]
            dataToWrite += "," + str(numberOfCommittersLR) #6. Committers Last Release (CtrLR) ! Number of committers on files for last release

        sqliteSTR = "SELECT filePath, fileLocAdd, fileLocRem from fileTable where projectName='" + projectName + "' and projectVersion='" + projectVersion + "' and commitDate between '" + str(lastReleaseDate) + "' and '" + str(endDate) + "'"
        cursor.execute(sqliteSTR)

        testDictLR = {}
        for row in cursor:
            if row[0] in testDictLR:
                testDictLR[row[0]]["added"] += row[1] # if key already exists, sum
                testDictLR[row[0]]["removed"] += row[2] # if key already exists, sum
            else:
                testDictLR[row[0]] = {}
                testDictLR[row[0]]["added"] = row[1] # else create a new key
                testDictLR[row[0]]["removed"] = row[2] # else create a new key

        if key in testDictLR:
            dataToWrite += "," + str(testDictLR[key]["added"]) #7. Lines Added Last Release (LadLR) ! Lines added on files for last release
        if key not in testDictLR:
            dataToWrite += "," + str(0)
        
        if key in testDictLR:
            dataToWrite += "," + str(testDictLR[key]["removed"]) #8. Lines Removed Last Release (LrmLR) ! Lines added on files for last release
        if key not in testDictLR:
            dataToWrite += "," + str(0)
        
        events = []
        sqliteSTR = "SELECT distinct(authorName) from fileTable where projectName='" + projectName + "' and projectVersion='" + projectVersion + "' and filePath = '" + key + "' and commitDate between '" + str(startDate) + "' and '" + str(endDate) + "'"
        cursor.execute(sqliteSTR)
        for row in cursor:
            events.append(row[0])

        occurrences = []
        sqliteSTR = "SELECT authorName from fileTable where projectName='" + projectName + "' and projectVersion='" + projectVersion + "' and filePath = '" + key + "' and commitDate between '" + str(startDate) + "' and '" + str(endDate) + "'"
        cursor.execute(sqliteSTR)    
        for row in cursor:
            occurrences.append(row[0])

        gini = 0    
        count = []    

        for i in range(len(events)):
            count.append(occurrences.count(events[i]))
    
        mean = len(occurrences) / float(len(events)) 
    
        for i in range(len(events)):
            for j in range(len(events)):
                gini += ( 1 / float(len(events) * len(events)))*abs( count[i] - count[j])
        giniIneq = gini/float(mean)

        dataToWrite += ",%.3f" % giniIneq #9. Gini Inequality Coefficient (GINI)!Commit inequality coefficient of committers on files

        if numberOfCommitters == 1:              #10. Top Committer Percentage (TCPer) ! Percentage of top committers on files
            dataToWrite += "," + str(100)
        elif not numberOfCommitters == 1:
            topCommiterPercentage = {}
            sqliteSTR = "SELECT authorName from fileTable where projectName='" + projectName + "' and projectVersion='" + projectVersion + "' and filePath = '" + key + "' and commitDate between '" + str(startDate) + "' and '" + str(endDate) + "'"
            cursor.execute(sqliteSTR)

            for row in cursor:
                if row[0] in topCommiterPercentage:
                    topCommiterPercentage[row[0]] += 1 # if key already exists, sum
                else:
                    topCommiterPercentage[row[0]] = 1 # else create a new key

            topCommiterCommit = 0

            for key in topCommiterPercentage:
                if topCommiterPercentage[key] > topCommiterCommit:
                    topCommiterCommit = topCommiterPercentage[key]
    
            dataToWrite += ",%.3f" % ((float(topCommiterCommit)/float(numberOfCommits))*100)            

        dataToWrite += "\n"

    FILE = open(filename,"w")
    FILE.writelines(dataToWrite)
    FILE.close()

    browser.append("\nThe test.csv File is CREATED...")
    QCoreApplication.processEvents(QEventLoop.ExcludeUserInputEvents)

    os.chdir(defaultDirectory)

def returnValuesforGKeys():
    dbFile = "commits.db"
    outputFile = "BugKeys.txt"
    projName = projectName
    projVersion = projectVersion
    tmpStartDate = trainStartDate.split(" ")
    sDate = tmpStartDate[0]
    tmpEndDate = trainEndDate.split(" ")
    eDate = tmpEndDate[0]

    return dbFile, outputFile, projName, projVersion, sDate, eDate
    

def generateKeys(dbFile,outputFile,projName,projVersion,sDate,eDate,browser):
    wordList=["bug","bugs","issue","problem","fix","fixes","fixed","land","landed","handle","handled"]
    typicalWordList=["java","status","perl","python","c","cpp","i","that","am","is","are","was","were","and","or","by","with","for","in","of","on","a","the","to","from","when","where","while","since","as","it","if"]
	
    startDate = datetime.strptime(sDate, '%Y-%m-%d')
    endDate = datetime.strptime(eDate, '%Y-%m-%d')
	
    cursor.execute("SELECT revisionNo,commitMessage FROM commitTable where projectName=? and projectVersion=? and commitDate between ? and ?", (projName,projVersion,str(startDate),str(endDate)))

    commitDict={}
    keyDict={}

    for row in cursor:
	commitDict[row[0]]=row[1]

    os.chdir("../../")
                
    reportFileName = "%s" % (outputFile)
    report= open(reportFileName,'w')

    for commitNo,commitMessage in commitDict.iteritems():
	for word in wordList:
	    if word in commitMessage:
		wordsInMessage=commitMessage.split(' ');
		for key in wordsInMessage:
		    if key in keyDict:
			keyDict[key] +=1
		    else: 
			keyDict[key]=1
		break

    uniqueList = {}
    for keyName,keyCount in keyDict.iteritems():
	if keyCount > 0 and keyName in wordList:
            keyName = re.sub(r'\W+', '', keyName)
            if keyName not in uniqueList:
                uniqueList[keyName] = keyName

    for key in sorted(uniqueList):
        buggyWord = key + "\n"
        report.write(buggyWord)

    report.close()
    browser.append("\n\nBugKeys file is created...")
    QCoreApplication.processEvents(QEventLoop.ExcludeUserInputEvents)

    wantToEdit = QtGui.QMessageBox.question(None, '..::Confirmation::..', "Want to EDIT BugKeys?!.", QtGui.QMessageBox.Yes, QtGui.QMessageBox.No)
    if wantToEdit == QtGui.QMessageBox.Yes:
        editKeys = editBugKeys()
        
        fileOpen = open(reportFileName,'r')
        lines = fileOpen.readlines()
        for line in lines:
            line = line.rstrip("\r\n")
            editKeys.writeToBrowser(line)
        fileOpen.close()
        
        editKeys.exec_()

        browser.append("\n\nBugKeys file is reviewed...")
        QCoreApplication.processEvents(QEventLoop.ExcludeUserInputEvents)


def returnValuesforDMatch():
    dbFile = "commits.db"
    keywordFile = "BugKeys.txt"
    outputFile = "BuggyFiles.txt"
    projName = projectName
    projVersion = projectVersion
    tmpStartDate = trainStartDate.split(" ")
    sDate = tmpStartDate[0]
    tmpEndDate = trainEndDate.split(" ")
    eDate = tmpEndDate[0]

    return dbFile, keywordFile, outputFile, projName, projVersion, sDate, eDate

def defectMatch(dbFile,keywordFile,outputFile,projName,projVersion,sDate,eDate,browser):
    wordList=["bug","issue","problem","fixed","fix","landed","land","solved","solve","corrected","correct","tweaked","tweak"]
	
    startDate = datetime.strptime(sDate, '%Y-%m-%d')
    endDate = datetime.strptime(eDate, '%Y-%m-%d')

    filename= "%s" % (keywordFile)
    f=open(filename, 'r')
    lines= f.readlines()
    f.close()
    for line in lines:
	if len(line.split('\n')[0]) > 0 :
	    wordList.append(line.split('\n')[0])

    cursor.execute("SELECT revisionNo,commitMessage FROM commitTable where projectName=? and projectVersion=? and commitDate between ? and ?", (projName,projVersion,str(startDate),str(endDate)))

    commitDict={}

    for row in cursor:
	commitDict[row[0]]=row[1]

    reportFileName = "%s" % (outputFile)
    report= open(reportFileName,'w')

    uniqueList = {}
    for commitNo,commitMessage in commitDict.iteritems():
	for word in wordList:
	    if word in commitMessage:
		cursor.execute("SELECT filePath FROM fileTable WHERE revisionNo=? and projectName=? and projectVersion=?",(commitNo,projName,projVersion))
		for row in cursor:
                    if row[0] not in uniqueList:
                        uniqueList[row[0]] = row[0]		    
		break

    for key in sorted(uniqueList):
        fileName = key + "\n"
        report.write(fileName)
        
    report.close()
    browser.append("\n\nBuggyFiles file is created...")
    QCoreApplication.processEvents(QEventLoop.ExcludeUserInputEvents)
    
def returnValuesforFMatch():
    fMetric = os.path.abspath('./' + projectName + "/" + projectVersion + '/train/train.csv')
    fBug = os.path.abspath('BuggyFiles.txt')

    return fMetric, fBug

def fileMatch(fMetric,fBug,browser):    
    fMetricDict = {}
    fBugDict = {}

    f1=open(fMetric, 'r')
    lines1= f1.readlines()
    f1.close()

    lines1.pop(0)
    
    for line1 in lines1:
	if len(line1) > 2:
	    fMetricDict[line1.split(',')[0]]=line1;
	
    f2=open(fBug, 'r')
    lines2= f2.readlines()
    f2.close()

    for line2 in lines2:
	if len(line2) > 2:
	    tracfilename=line2.split(',')[0]
	    fBugDict[tracfilename.split('\n')[0]]=1

    os.chdir("./" + projectName + "/" + projectVersion + "/train")

    fileTitle = "File Name,Number Of Commits,Number Of Committers,Lines Added,Lines Removed,Number Of Commits(LR),Number Of Committers(LR),Lines Added(LR),Lines Removed(LR),Gini Inequality,Top Committer Percentage,Defected?(false/true)\n"

    reportFileName = "trainFile.csv"
    report= open(reportFileName,'w')

    report.write(fileTitle)
    
    for filename,metrics in fMetricDict.iteritems():
	filename=filename.strip("\n")
	if filename.strip("\n") in fBugDict:
	    metrics=metrics.strip()
	    report.write("%s,%s\n" % (metrics,"true"))			
	else:	
	    metrics=metrics.strip()
	    report.write("%s,%s\n" % (metrics,"false"))

    browser.append("\n\nFile Match is done...")
    QCoreApplication.processEvents(QEventLoop.ExcludeUserInputEvents)
    os.chdir("../../../")

def returnPathforProject():
    prestProjName = projectName
    prestProjVersion = projectVersion

    return prestProjName, prestProjVersion

def addProject(prestProjName, prestProjRootDir, repoDir, browser):
    browser.clear()

    os.chdir("../")

    if not os.path.exists("./" + repoDir + "/"):
        os.mkdir("./" + repoDir + "/")
        browser.append("Prest Repository created...\n\n")
        QCoreApplication.processEvents(QEventLoop.ExcludeUserInputEvents)
        
    proc = Popen("java -jar Prest.jar -addProject projectRoot/" + str(prestProjRootDir), shell=True, stdout=PIPE)
    while True:
        out = proc.stdout.readline()
        if out == '' and proc.poll() != None:
            break
        if out != '':
            out = out.rstrip("\r\n")
            browser.append(out)
        QCoreApplication.processEvents(QEventLoop.ExcludeUserInputEvents)
    
    browser.append("\n\nProject is added into Prest...\n\n")
    QCoreApplication.processEvents(QEventLoop.ExcludeUserInputEvents)
        
def parseProject(prestProjName, prestProjRootDir, prestProjVersion, browser):
    proc = Popen("java -jar Prest.jar -parse projectRoot/" + str(prestProjRootDir) + " " + prestProjVersion, shell=True, stdout=PIPE)
    while True:
        out = proc.stdout.readline()
        if out == '' and proc.poll() != None:
            break
        if out != '':
            out = out.rstrip("\r\n")
            browser.append(out)
        QCoreApplication.processEvents(QEventLoop.ExcludeUserInputEvents)
            
    browser.append("\n<< Project: " + prestProjRootDir + " is parsed... >>")
    QCoreApplication.processEvents(QEventLoop.ExcludeUserInputEvents)


def copyFilesInParseResultsFolder(prestProjRootDir, repoDir):
    dir_name = os.path.abspath('./'+ repoDir + "/" + str(prestProjRootDir) + "/" + "parse_results")

    trainCsvFile = os.path.abspath('./projectRoot' + "/" + projectName + "/" + projectVersion + "/" + "train" + "/" + "trainFile.csv")
    testCsvFile = os.path.abspath('./projectRoot' + "/" + projectName + "/" + projectVersion + "/" + "test" + "/" + "test.csv")
    
    os.system("move " + str(trainCsvFile) + " " + str(dir_name))
    os.system("move " + str(testCsvFile) + " " + str(dir_name))

    pathToRemove = os.path.abspath('./' + "projectRoot/" + str(prestProjRootDir) + "/tempWord")
    pathToRemove = pathToRemove.replace("tempWord", "")

    prestOutputFile = ""
    for outputFile in os.listdir(dir_name):
        if outputFile.find("FILE") > 0:
            prestOutputFile = outputFile

    defaultDirectory = os.getcwd()

    os.chdir(dir_name)
 
    fileNo1 = open(prestOutputFile, 'r')
    rows = fileNo1.readlines()
    fileNo1.close()

    columnsToWrite = ""
    for columns in rows:
        columnsData = columns.split(',')
        newFileName = columnsData[0].replace(pathToRemove,"")
        i = 1
        columnsToWrite += newFileName + ","
        while i < len(columnsData):
            columnsToWrite += columnsData[i]
            i += 1
            if i != len(columnsData):
                columnsToWrite +=","

    finalOutputFile = open("parseOutputFile.csv",'w')
    finalOutputFile.write(columnsToWrite)            
    finalOutputFile.close()
                
    os.chdir(defaultDirectory)     

def prestFileMatch(prestProjRootDir, repoDir, browser):
    prestOutputFile = os.path.abspath('./' + str(repoDir) + "/" + str(prestProjRootDir) + "/" + "parse_results" + "/" + "parseOutputFile.csv")
    trainFile = os.path.abspath('./' + str(repoDir) + "/" + str(prestProjRootDir) + "/" + "parse_results" + "/" + "trainFile.csv")
    testFile = os.path.abspath('./' + str(repoDir) + "/" + str(prestProjRootDir) + "/" + "parse_results" + "/" + "test.csv")
    
    outputTrainFile = "finalTrain.csv"
    outputTestFile = "finalTest.csv"    
    tempTestFile = "tempTest.csv"    

    fileNo1 = open(prestOutputFile, 'r')
    row1 = fileNo1.readlines()
    fileNo1.close()

    fileNo2=open(trainFile, 'r')
    row2= fileNo2.readlines()
    fileNo2.close()
    
    fileTitle = ""
    rowToWrite = ""
    finalRowToWrite = ""
    
    fileTitle1 = row1.pop(0).rstrip("\r\n").split(',')

    i = 0
    while i < (len(fileTitle1)-1):
        fileTitle += fileTitle1[i] + ","
        i += 1

    
    fileTitle2 = row2.pop(0).rstrip("\r\n").split(',')

    i = 1

    while i < (len(fileTitle2)-1):
        fileTitle += fileTitle2[i] + ","
        i += 1
    fileTitle += fileTitle1[(len(fileTitle1)-1)] + "\n"
    
    for column1 in sorted(row1):
        column1Data = column1.split(',')
        column1Data.pop(22)

        existsInTrain = "false"
        
        for column2 in sorted(row2):
            column2Data = column2.split(',')
            if column1Data[0] == column2Data[0]:
                existsInTrain = "true"
                i = 0
                while i < len(column1Data):
                    rowToWrite += column1Data[i] + ","
                    i = i + 1
                    if i == len(column1Data):
                        j = 1
                        while j < (len(column2Data)-1):
                            rowToWrite += column2Data[j] + ","
                            j += 1
                            if (j+1) == len(column2Data):
                                if column2Data[(len(column2Data)-1)].rstrip("\r\n").lower() == "true":
                                    rowToWrite += "TRUE\n"
                                else:
                                    rowToWrite += "FALSE\n"
                
        if existsInTrain == "false":
            i = 0
            while i < len(column1Data):
                rowToWrite += column1Data[i] + ","
                i = i + 1
                if i == len(column1Data):
                    j = 1
                    while j < (len(column2Data)-1):
                        rowToWrite += "0" + ","
                        j += 1
                        if (j+1) == len(column2Data):
                            rowToWrite += "FALSE\n"

    finalRowToWrite = fileTitle + rowToWrite
    finalTrainFile = open(outputTrainFile,'w')
    finalTrainFile.write(finalRowToWrite)            
    finalTrainFile.close()
    


    fileNo3 = open(prestOutputFile, 'r')
    row3 = fileNo3.readlines()
    fileNo3.close()

    fileNo4=open(testFile, 'r')
    row4= fileNo4.readlines()
    fileNo4.close()
    
    fileTitle = ""
    rowToWrite = ""
    finalRowToWrite = ""
    
    fileTitle3 = row3.pop(0).rstrip("\r\n").split(',')

    i = 0
    while i < (len(fileTitle3)-1):
        fileTitle += fileTitle3[i] + ","
        i += 1

    
    fileTitle4 = row4.pop(0).rstrip("\r\n").split(',')

    i = 1

    while i < len(fileTitle4):
        fileTitle += fileTitle4[i] + ","
        i += 1
    fileTitle += fileTitle3[(len(fileTitle3)-1)] + "\n"
    
    for column3 in sorted(row3):
        column3Data = column3.split(',')
        column3Data.pop(22)

        existsInTrain = "false"
        
        for column4 in sorted(row4):
            column4Data = column4.split(',')
            if column3Data[0] == column4Data[0]:
                existsInTrain = "true"
                i = 0
                while i < len(column3Data):
                    rowToWrite += column3Data[i] + ","
                    i = i + 1
                    if i == len(column3Data):
                        j = 1
                        while j < len(column4Data):
                            rowToWrite += column4Data[j].rstrip("\r\n") + ","
                            j += 1
                            if j == len(column4Data):
                                if column4Data[(len(column4Data)-1)].rstrip("\r\n").lower() == "true":
                                    rowToWrite += "TRUE\n"
                                else:
                                    rowToWrite += "FALSE\n"
                
        if existsInTrain == "false":
            i = 0
            while i < len(column3Data):
                rowToWrite += column3Data[i] + ","
                i = i + 1
                if i == len(column3Data):
                    j = 1
                    while j < len(column4Data):
                        rowToWrite += "0" + ","
                        j += 1
                        if j == len(column4Data):
                            rowToWrite += "FALSE\n"

    finalRowToWrite = fileTitle + rowToWrite
    finalTestFile = open(tempTestFile,'w')
    finalTestFile.write(finalRowToWrite)            
    finalTestFile.close()



    fileTitle = ""
    rowToWrite = ""
    finalRowToWrite = ""
    
    fileNo5 = open(outputTrainFile, 'r')
    row5 = fileNo5.readlines()
    fileNo5.close()

    fileTitle = row5.pop(0)

    fileNo6=open(tempTestFile, 'r')
    row6= fileNo6.readlines()
    fileNo6.close()

    row6.pop(0)

    for column5 in sorted(row5):
        column5Data = column5.split(',')
        if column5Data[(len(column5Data)-1)].rstrip("\r\n").lower()=="true":
            for column6 in sorted(row6):
                column6Data = column6.split(',')
                if column5Data[0] == column6Data[0]:
                    i = 0
                    while i < (len(column6Data)-1):
                        rowToWrite += column6Data[i] + ","
                        i += 1
                        if (i+1) == len(column6Data):
                            rowToWrite += "TRUE\n"
        else:
            for column6 in sorted(row6):
                column6Data = column6.split(',')
                if column5Data[0] == column6Data[0]:
                    i = 0
                    while i < (len(column6Data)-1):
                        rowToWrite += column6Data[i] + ","
                        i += 1
                        if (i+1) == len(column6Data):
                            rowToWrite += "FALSE\n"            

    finalRowToWrite = fileTitle + rowToWrite
    finalTestFile = open(outputTestFile,'w')
    finalTestFile.write(finalRowToWrite)            
    finalTestFile.close() 

    dir_name = os.path.abspath('./'+ repoDir + "/" + str(prestProjRootDir) + "/" + "parse_results")

    trainCsvFilePath = os.path.abspath('./finalTrain.csv')
    testCsvFilePath = os.path.abspath('./finalTest.csv')
    tempTestCsvFilePath = os.path.abspath('./tempTest.csv')

    os.system("del " + str(tempTestCsvFilePath))
    os.system("move " + str(trainCsvFilePath) + " " + str(dir_name))
    os.system("move " + str(testCsvFilePath) + " " + str(dir_name))

    browser.append("\n\nFiles are matched with prest parse output...\n\n")
    QCoreApplication.processEvents(QEventLoop.ExcludeUserInputEvents)

def convertToArff(prestProjRootDir, repoDir, browser):
    trainCsvFile = os.path.abspath(str(repoDir) + "/" + str(prestProjRootDir) + "/" + "parse_results" + "/" + "finalTrain.csv")

    proc = Popen("java -jar Prest.jar -convertCsvToArff " + str(trainCsvFile), shell=True, stdout=PIPE)
    while True:
        out = proc.stdout.readline()
        if out == '' and proc.poll() != None:
            break
        if out != '':
            out = out.rstrip("\r\n")
            browser.append(out)
        QCoreApplication.processEvents(QEventLoop.ExcludeUserInputEvents)
            
    browser.append("\n<< trainFile.csv is converted to Arff... >>\n\n")
    QCoreApplication.processEvents(QEventLoop.ExcludeUserInputEvents)
    
    
    testCsvFile = os.path.abspath(str(repoDir) + "/" + str(prestProjRootDir) + "/" + "parse_results" + "/" + "finalTest.csv")
    
    proc = Popen("java -jar Prest.jar -convertCsvToArff " + str(testCsvFile), shell=True, stdout=PIPE)
    while True:
        out = proc.stdout.readline()
        if out == '' and proc.poll() != None:
            break
        if out != '':
            out = out.rstrip("\r\n")
            browser.append(out)
        QCoreApplication.processEvents(QEventLoop.ExcludeUserInputEvents)
            
    browser.append("\n<< test.csv is converted to Arff... >>\n\n")
    QCoreApplication.processEvents(QEventLoop.ExcludeUserInputEvents)

def predictWithPrest(prestProjRootDir, repoDir, browser):
    trainArffFile = os.path.abspath(str(repoDir) + "/" + str(prestProjRootDir) + "/" + "arff_files" + "/" + "finalTrain.arff")
    testArffFile = os.path.abspath(str(repoDir) + "/" + str(prestProjRootDir) + "/" + "arff_files" + "/" + "finalTest.arff")
    
    proc = Popen("java -jar Prest.jar -predict " + str(trainArffFile) + " " + str(testArffFile), shell=True, stdout=PIPE)
    while True:
        out = proc.stdout.readline()
        if out == '' and proc.poll() != None:
            break
        if out != '':
            out = out.rstrip("\r\n")
            browser.append(out)
        QCoreApplication.processEvents(QEventLoop.ExcludeUserInputEvents)
        
    browser.append("\nPredict is done...")
    QCoreApplication.processEvents(QEventLoop.ExcludeUserInputEvents)
